{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 环境配置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%capture captured_output\n",
    "# 实验环境已经预装了mindspore==2.3.0，如需更换mindspore版本，可更改下面 MINDSPORE_VERSION 变量\n",
    "!pip uninstall mindspore -y\n",
    "!export MINDSPORE_VERSION=2.3.0\n",
    "!pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${MINDSPORE_VERSION}/MindSpore/unified/aarch64/mindspore-${MINDSPORE_VERSION}-cp39-cp39-linux_aarch64.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.mirrors.ustc.edu.cn/simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple\n",
      "Requirement already satisfied: mindnlp in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (0.3.1)\n",
      "Requirement already satisfied: mindspore in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (2.3.0)\n",
      "Requirement already satisfied: tqdm in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (4.66.4)\n",
      "Requirement already satisfied: requests in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (2.32.3)\n",
      "Requirement already satisfied: datasets in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (2.20.0)\n",
      "Requirement already satisfied: evaluate in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.4.2)\n",
      "Requirement already satisfied: tokenizers in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.19.1)\n",
      "Requirement already satisfied: safetensors in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.4.3)\n",
      "Requirement already satisfied: sentencepiece in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.2.0)\n",
      "Requirement already satisfied: regex in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (2024.5.15)\n",
      "Requirement already satisfied: addict in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (2.4.0)\n",
      "Requirement already satisfied: ml-dtypes in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.4.0)\n",
      "Requirement already satisfied: pyctcdecode in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.5.0)\n",
      "Requirement already satisfied: jieba in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (0.42.1)\n",
      "Requirement already satisfied: pytest==7.2.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindnlp) (7.2.0)\n",
      "Requirement already satisfied: attrs>=19.2.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (23.2.0)\n",
      "Requirement already satisfied: iniconfig in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (2.0.0)\n",
      "Requirement already satisfied: packaging in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (24.1)\n",
      "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (1.5.0)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (1.2.0)\n",
      "Requirement already satisfied: tomli>=1.0.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pytest==7.2.0->mindnlp) (2.0.1)\n",
      "Requirement already satisfied: filelock in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (3.15.4)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (1.26.4)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (16.1.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (0.3.8)\n",
      "Requirement already satisfied: pandas in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (2.2.2)\n",
      "Requirement already satisfied: xxhash in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.5.0,>=2023.1.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from fsspec[http]<=2024.5.0,>=2023.1.0->datasets->mindnlp) (2024.5.0)\n",
      "Requirement already satisfied: aiohttp in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (3.9.5)\n",
      "Requirement already satisfied: huggingface-hub>=0.21.2 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (0.24.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from datasets->mindnlp) (6.0.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from requests->mindnlp) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from requests->mindnlp) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from requests->mindnlp) (2.2.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from requests->mindnlp) (2024.7.4)\n",
      "Requirement already satisfied: protobuf>=3.13.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (5.27.2)\n",
      "Requirement already satisfied: asttokens>=2.0.4 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (2.0.5)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (10.4.0)\n",
      "Requirement already satisfied: scipy>=1.5.4 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (1.13.1)\n",
      "Requirement already satisfied: psutil>=5.6.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (5.9.0)\n",
      "Requirement already satisfied: astunparse>=1.6.3 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from mindspore->mindnlp) (1.6.3)\n",
      "Requirement already satisfied: pygtrie<3.0,>=2.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pyctcdecode->mindnlp) (2.5.0)\n",
      "Requirement already satisfied: hypothesis<7,>=6.14 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pyctcdecode->mindnlp) (6.105.1)\n",
      "Requirement already satisfied: six in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from asttokens>=2.0.4->mindspore->mindnlp) (1.16.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from astunparse>=1.6.3->mindspore->mindnlp) (0.43.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from aiohttp->datasets->mindnlp) (1.3.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from aiohttp->datasets->mindnlp) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from aiohttp->datasets->mindnlp) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from aiohttp->datasets->mindnlp) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from aiohttp->datasets->mindnlp) (4.0.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from huggingface-hub>=0.21.2->datasets->mindnlp) (4.11.0)\n",
      "Requirement already satisfied: sortedcontainers<3.0.0,>=2.1.0 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from hypothesis<7,>=6.14->pyctcdecode->mindnlp) (2.4.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pandas->datasets->mindnlp) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pandas->datasets->mindnlp) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages (from pandas->datasets->mindnlp) (2024.1)\n"
     ]
    }
   ],
   "source": [
    "# 该案例在 mindnlp 0.3.1 版本完成适配，如果发现案例跑不通，可以指定mindnlp版本，执行`!pip install mindnlp==0.3.1`\n",
    "!pip install mindnlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: mindspore\n",
      "Version: 2.3.0\n",
      "Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.\n",
      "Home-page: https://www.mindspore.cn\n",
      "Author: The MindSpore Authors\n",
      "Author-email: contact@mindspore.cn\n",
      "License: Apache 2.0\n",
      "Location: /home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages\n",
      "Requires: asttokens, astunparse, numpy, packaging, pillow, protobuf, psutil, scipy\n",
      "Required-by: mindnlp\n"
     ]
    }
   ],
   "source": [
    "!pip show mindspore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于 MindSpore 实现 BERT 对话情绪识别\n",
    "\n",
    "\n",
    "## 模型简介\n",
    "\n",
    "BERT全称是来自变换器的双向编码器表征量（Bidirectional Encoder Representations from Transformers），它是Google于2018年末开发并发布的一种新型语言模型。与BERT模型相似的预训练语言模型例如问答、命名实体识别、自然语言推理、文本分类等在许多自然语言处理任务中发挥着重要作用。模型是基于Transformer中的Encoder并加上双向的结构，因此一定要熟练掌握Transformer的Encoder的结构。\n",
    "\n",
    "BERT模型的主要创新点都在pre-train方法上，即用了Masked Language Model和Next Sentence Prediction两种方法分别捕捉词语和句子级别的representation。\n",
    "\n",
    "在用Masked Language Model方法训练BERT的时候，随机把语料库中15%的单词做Mask操作。对于这15%的单词做Mask操作分为三种情况：80%的单词直接用[Mask]替换、10%的单词直接替换成另一个新的单词、10%的单词保持不变。\n",
    "\n",
    "因为涉及到Question Answering (QA) 和 Natural Language Inference (NLI)之类的任务，增加了Next Sentence Prediction预训练任务，目的是让模型理解两个句子之间的联系。与Masked Language Model任务相比，Next Sentence Prediction更简单些，训练的输入是句子A和B，B有一半的几率是A的下一句，输入这两个句子，BERT模型预测B是不是A的下一句。\n",
    "\n",
    "BERT预训练之后，会保存它的Embedding table和12层Transformer权重（BERT-BASE）或24层Transformer权重（BERT-LARGE）。使用预训练好的BERT模型可以对下游任务进行Fine-tuning，比如：文本分类、相似度判断、阅读理解等。\n",
    "\n",
    "对话情绪识别（Emotion Detection，简称EmoTect），专注于识别智能对话场景中用户的情绪，针对智能对话场景中的用户文本，自动判断该文本的情绪类别并给出相应的置信度，情绪类型分为积极、消极、中性。 对话情绪识别适用于聊天、客服等多个场景，能够帮助企业更好地把握对话质量、改善产品的用户交互体验，也能分析客服服务质量、降低人工质检成本。\n",
    "\n",
    "下面以一个文本情感分类任务为例子来说明BERT模型的整个应用过程。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages/numpy/core/getlimits.py:549: UserWarning: The value of the smallest subnormal for <class 'numpy.float64'> type is zero.\n",
      "  setattr(self, word, getattr(machar, word).flat[0])\n",
      "/home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for <class 'numpy.float64'> type is zero.\n",
      "  return self._float_to_str(self.smallest_subnormal)\n",
      "/home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages/numpy/core/getlimits.py:549: UserWarning: The value of the smallest subnormal for <class 'numpy.float32'> type is zero.\n",
      "  setattr(self, word, getattr(machar, word).flat[0])\n",
      "/home/mindspore/miniconda/envs/jupyter/lib/python3.9/site-packages/numpy/core/getlimits.py:89: UserWarning: The value of the smallest subnormal for <class 'numpy.float32'> type is zero.\n",
      "  return self._float_to_str(self.smallest_subnormal)\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 1.034 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "import mindspore\n",
    "from mindspore.dataset import text, GeneratorDataset, transforms\n",
    "from mindspore import nn, context\n",
    "\n",
    "from mindnlp._legacy.engine import Trainer, Evaluator\n",
    "from mindnlp._legacy.engine.callbacks import CheckpointCallback, BestModelCallback\n",
    "from mindnlp._legacy.metrics import Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# prepare dataset\n",
    "class SentimentDataset:\n",
    "    \"\"\"Sentiment Dataset\"\"\"\n",
    "\n",
    "    def __init__(self, path):\n",
    "        self.path = path\n",
    "        self._labels, self._text_a = [], []\n",
    "        self._load()\n",
    "\n",
    "    def _load(self):\n",
    "        with open(self.path, \"r\", encoding=\"utf-8\") as f:\n",
    "            dataset = f.read()\n",
    "        lines = dataset.split(\"\\n\")\n",
    "        for line in lines[1:-1]:\n",
    "            label, text_a = line.split(\"\\t\")\n",
    "            self._labels.append(int(label))\n",
    "            self._text_a.append(text_a)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self._labels[index], self._text_a[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据集\n",
    "\n",
    "这里提供一份已标注的、经过分词预处理的机器人聊天数据集，来自于百度飞桨团队。数据由两列组成，以制表符（'\\t'）分隔，第一列是情绪分类的类别（0表示消极；1表示中性；2表示积极），第二列是以空格分词的中文文本，如下示例，文件为 utf8 编码。\n",
    "\n",
    "label--text_a\n",
    "\n",
    "0--谁骂人了？我从来不骂人，我骂的都不是人，你是人吗 ？\n",
    "\n",
    "1--我有事等会儿就回来和你聊\n",
    "\n",
    "2--我见到你很高兴谢谢你帮我\n",
    "\n",
    "这部分主要包括数据集读取，数据格式转换，数据 Tokenize 处理和 pad 操作。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-07-31 10:32:11--  https://baidu-nlp.bj.bcebos.com/emotion_detection-dataset-1.0.0.tar.gz\n",
      "Resolving baidu-nlp.bj.bcebos.com (baidu-nlp.bj.bcebos.com)... 113.200.2.111, 119.249.103.5, 2409:8c04:1001:1203:0:ff:b0bb:4f27\n",
      "Connecting to baidu-nlp.bj.bcebos.com (baidu-nlp.bj.bcebos.com)|113.200.2.111|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1710581 (1.6M) [application/x-gzip]\n",
      "Saving to: ‘emotion_detection.tar.gz’\n",
      "\n",
      "emotion_detection.t 100%[===================>]   1.63M  3.38MB/s    in 0.5s    \n",
      "\n",
      "2024-07-31 10:32:12 (3.38 MB/s) - ‘emotion_detection.tar.gz’ saved [1710581/1710581]\n",
      "\n",
      "data/\n",
      "data/test.tsv\n",
      "data/infer.tsv\n",
      "data/dev.tsv\n",
      "data/train.tsv\n",
      "data/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "# download dataset\n",
    "!wget https://baidu-nlp.bj.bcebos.com/emotion_detection-dataset-1.0.0.tar.gz -O emotion_detection.tar.gz\n",
    "!tar xvf emotion_detection.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据加载和数据预处理\n",
    "\n",
    "新建 process_dataset 函数用于数据加载和数据预处理，具体内容可见下面代码注释。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def process_dataset(source, tokenizer, max_seq_len=64, batch_size=32, shuffle=True):\n",
    "    is_ascend = mindspore.get_context('device_target') == 'Ascend'\n",
    "\n",
    "    column_names = [\"label\", \"text_a\"]\n",
    "    \n",
    "    dataset = GeneratorDataset(source, column_names=column_names, shuffle=shuffle)\n",
    "    # transforms\n",
    "    type_cast_op = transforms.TypeCast(mindspore.int32)\n",
    "    def tokenize_and_pad(text):\n",
    "        if is_ascend:\n",
    "            tokenized = tokenizer(text, padding='max_length', truncation=True, max_length=max_seq_len)\n",
    "        else:\n",
    "            tokenized = tokenizer(text)\n",
    "        return tokenized['input_ids'], tokenized['attention_mask']\n",
    "    # map dataset\n",
    "    dataset = dataset.map(operations=tokenize_and_pad, input_columns=\"text_a\", output_columns=['input_ids', 'attention_mask'])\n",
    "    dataset = dataset.map(operations=[type_cast_op], input_columns=\"label\", output_columns='labels')\n",
    "    # batch dataset\n",
    "    if is_ascend:\n",
    "        dataset = dataset.batch(batch_size)\n",
    "    else:\n",
    "        dataset = dataset.padded_batch(batch_size, pad_info={'input_ids': (None, tokenizer.pad_token_id),\n",
    "                                                         'attention_mask': (None, 0)})\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "昇腾NPU环境下暂不支持动态Shape，数据预处理部分采用静态Shape处理："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f7d528aeb8e4123b76ae8db3b90610d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/49.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e52005b5ca94492a0e1c69a9484f0ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d78cfc43aa36400c98881ee863769e7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "348062cd65e440fa8a7fad173855060a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/324 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from mindnlp.transformers import BertTokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset_train = process_dataset(SentimentDataset(\"data/train.tsv\"), tokenizer)\n",
    "dataset_val = process_dataset(SentimentDataset(\"data/dev.tsv\"), tokenizer)\n",
    "dataset_test = process_dataset(SentimentDataset(\"data/test.tsv\"), tokenizer, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['input_ids', 'attention_mask', 'labels']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_train.get_col_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Tensor(shape=[32, 64], dtype=Int64, value=\n",
      "[[ 101,  714, 2209 ...    0,    0,    0],\n",
      " [ 101,  872, 1435 ...    0,    0,    0],\n",
      " [ 101,  872, 2582 ...    0,    0,    0],\n",
      " ...\n",
      " [ 101,  872, 7309 ...    0,    0,    0],\n",
      " [ 101,  872,  812 ...    0,    0,    0],\n",
      " [ 101,  872, 4339 ...    0,    0,    0]]), Tensor(shape=[32, 64], dtype=Int64, value=\n",
      "[[1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " ...\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0]]), Tensor(shape=[32], dtype=Int32, value= [1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, \n",
      " 1, 1, 1, 1, 1, 1, 1, 1])]\n"
     ]
    }
   ],
   "source": [
    "print(next(dataset_train.create_tuple_iterator()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型构建\n",
    "\n",
    "通过 BertForSequenceClassification 构建用于情感分类的 BERT 模型，加载预训练权重，设置情感三分类的超参数自动构建模型。后面对模型采用自动混合精度操作，提高训练的速度，然后实例化优化器，紧接着实例化评价指标，设置模型训练的权重保存策略，最后就是构建训练器，模型开始训练。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59d42c3a745445e9852af8125df950ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/392M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following parameters in checkpoint files are not loaded:\n",
      "['cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "The following parameters in models are missing parameter:\n",
      "['classifier.weight', 'classifier.bias']\n"
     ]
    }
   ],
   "source": [
    "from mindnlp.transformers import BertForSequenceClassification, BertModel\n",
    "from mindnlp._legacy.amp import auto_mixed_precision\n",
    "\n",
    "# set bert config and define parameters for training\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=3)\n",
    "model = auto_mixed_precision(model, 'O1')\n",
    "\n",
    "optimizer = nn.Adam(model.trainable_params(), learning_rate=2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "metric = Accuracy()\n",
    "# define callbacks to save checkpoints\n",
    "ckpoint_cb = CheckpointCallback(save_path='checkpoint', ckpt_name='bert_emotect', epochs=1, keep_checkpoint_max=2)\n",
    "best_model_cb = BestModelCallback(save_path='checkpoint', ckpt_name='bert_emotect_best', auto_load=True)\n",
    "\n",
    "trainer = Trainer(network=model, train_dataset=dataset_train,\n",
    "                  eval_dataset=dataset_val, metrics=metric,\n",
    "                  epochs=5, optimizer=optimizer, callbacks=[ckpoint_cb, best_model_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The train will start from the checkpoint saved in 'checkpoint'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b89cd4211b9490ea39aa15b941e1529",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/302 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint: 'bert_emotect_epoch_0.ckpt' has been saved in epoch: 0.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d3464c2b0e449a5927b4857b0bf84bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/34 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9175925925925926}\n",
      "---------------Best Model: 'bert_emotect_best.ckpt' has been saved in epoch: 0.---------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96b26a23cbf14b9aaa04a85a9ec9bdf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/302 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint: 'bert_emotect_epoch_1.ckpt' has been saved in epoch: 1.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b54c33536cba4c41bfc0984132a57224",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/34 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9629629629629629}\n",
      "---------------Best Model: 'bert_emotect_best.ckpt' has been saved in epoch: 1.---------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb06b35099094b59a78745fde0a214ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/302 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum number of stored checkpoints has been reached.\n",
      "Checkpoint: 'bert_emotect_epoch_2.ckpt' has been saved in epoch: 2.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2046b5277c51409d927627b756774682",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/34 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9814814814814815}\n",
      "---------------Best Model: 'bert_emotect_best.ckpt' has been saved in epoch: 2.---------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ee507d7e95e4378ae385249a19665df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/302 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum number of stored checkpoints has been reached.\n",
      "Checkpoint: 'bert_emotect_epoch_3.ckpt' has been saved in epoch: 3.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1e1d544c2ab4712917ce54028ce66a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/34 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9861111111111112}\n",
      "---------------Best Model: 'bert_emotect_best.ckpt' has been saved in epoch: 3.---------------\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4365a284b7c24245a6cce267c2b06636",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/302 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The maximum number of stored checkpoints has been reached.\n",
      "Checkpoint: 'bert_emotect_epoch_4.ckpt' has been saved in epoch: 4.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b051be507bd4b8d835dacbe7573ccf6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/34 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.9675925925925926}\n",
      "Loading best model from 'checkpoint' with '['Accuracy']': [0.9861111111111112]...\n",
      "---------------The model is already load the best model from 'bert_emotect_best.ckpt'.---------------\n",
      "CPU times: user 43min 6s, sys: 10min 11s, total: 53min 18s\n",
      "Wall time: 12min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# start training\n",
    "trainer.run(tgt_columns=\"labels\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型验证\n",
    "\n",
    "将验证数据集加再进训练好的模型，对数据集进行验证，查看模型在验证数据上面的效果，此处的评价指标为准确率。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fab8865c5f8482db3ef98a33deba6b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/33 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate Score: {'Accuracy': 0.8938223938223938}\n"
     ]
    }
   ],
   "source": [
    "evaluator = Evaluator(network=model, eval_dataset=dataset_test, metrics=metric)\n",
    "evaluator.run(tgt_columns=\"labels\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型推理\n",
    "\n",
    "遍历推理数据集，将结果与标签进行统一展示。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset_infer = SentimentDataset(\"data/infer.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def predict(text, label=None):\n",
    "    label_map = {0: \"消极\", 1: \"中性\", 2: \"积极\"}\n",
    "\n",
    "    text_tokenized = Tensor([tokenizer(text).input_ids])\n",
    "    logits = model(text_tokenized)\n",
    "    predict_label = logits[0].asnumpy().argmax()\n",
    "    info = f\"inputs: '{text}', predict: '{label_map[predict_label]}'\"\n",
    "    if label is not None:\n",
    "        info += f\" , label: '{label_map[label]}'\"\n",
    "    print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs: '我 要 客观', predict: '中性' , label: '中性'\n",
      "inputs: '靠 你 真是 说 废话 吗', predict: '消极' , label: '消极'\n",
      "inputs: '口嗅 会', predict: '中性' , label: '中性'\n",
      "inputs: '每次 是 表妹 带 窝 飞 因为 窝路痴', predict: '中性' , label: '中性'\n",
      "inputs: '别说 废话 我 问 你 个 问题', predict: '消极' , label: '消极'\n",
      "inputs: '4967 是 新加坡 那 家 银行', predict: '中性' , label: '中性'\n",
      "inputs: '是 我 喜欢 兔子', predict: '积极' , label: '积极'\n",
      "inputs: '你 写 过 黄山 奇石 吗', predict: '中性' , label: '中性'\n",
      "inputs: '一个一个 慢慢来', predict: '中性' , label: '中性'\n",
      "inputs: '我 玩 过 这个 一点 都 不 好玩', predict: '消极' , label: '消极'\n",
      "inputs: '网上 开发 女孩 的 QQ', predict: '中性' , label: '中性'\n",
      "inputs: '背 你 猜 对 了', predict: '中性' , label: '中性'\n",
      "inputs: '我 讨厌 你 ， 哼哼 哼 。 。', predict: '消极' , label: '消极'\n"
     ]
    }
   ],
   "source": [
    "from mindspore import Tensor\n",
    "\n",
    "for label, text in dataset_infer:\n",
    "    predict(text, label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自定义推理数据集\n",
    "\n",
    "自己输入推理数据，展示模型的泛化能力。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs: '家人们咱就是说一整个无语住了 绝绝子叠buff', predict: '中性'\n"
     ]
    }
   ],
   "source": [
    "predict(\"家人们咱就是说一整个无语住了 绝绝子叠buff\")"
   ]
  }
 ],
 "metadata": {
  "AIGalleryInfo": {
   "item_id": "5443b528-0dd5-4909-ac4f-1c9cf839e2aa"
  },
  "flavorInfo": {
   "architecture": "X86_64",
   "category": "GPU"
  },
  "imageInfo": {
   "id": "e1a07296-22a8-4f05-8bc8-e936c8e54202",
   "name": "mindspore1.7.0-cuda10.1-py3.7-ubuntu18.04"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
